ELK 體驗

2025 iThome 鐵人賽

DAY 21

佛心分享-我的私藏工具箱

天花亂墜的走馬看花之軟體體驗之旅系列第 21 篇

17th鐵人賽

steven715

2025-10-05 20:30:11

59 瀏覽

分享至

今天來體驗 ELK ，實際上ELK是由 ElasticSearch(搜尋引擎)、Logstash(資料處理管道)、Kinbana(UI介紹提供搜尋結果)三種組成而成的。

今日目標

透過docker方式部署ELK，並且寫一個使用spdlog 的c++ 容器，讓log能被ELK接收，並讓再kibana 上面查詢

環境準備

C:\USERS\ASUS\STEVEN\ELK-LAB
│  docker-compose.yml
│  filebeat.yml
│
├─app-cpp
│  │  Dockerfile
│  │
│  └─cpp-spdlog
│          CMakeLists.txt
│          main.cpp
│
└─logs

docker-compose.yml

version: "3.9"
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3
    container_name: es
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false        # demo 簡化：先關掉 auth
      - ES_JAVA_OPTS=-Xms1g -Xmx1g
    ulimits:
      memlock: { soft: -1, hard: -1 }
    ports:
      - "9200:9200"
    volumes:
      - esdata:/usr/share/elasticsearch/data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"]
      interval: 10s
      timeout: 5s
      retries: 20

  kibana:
    image: docker.elastic.co/kibana/kibana:8.14.3
    container_name: kib
    environment:
      - ELASTICSEARCH_HOSTS=http://es:9200
      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
    ports:
      - "5601:5601"
    depends_on:
      - elasticsearch

  filebeat:
    image: docker.elastic.co/beats/filebeat:8.14.3
    container_name: fb
    user: root
    depends_on:
      - elasticsearch
      - kibana
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
      - /c/Users/Asus/steven/elk-lab/logs:/logs:ro
    environment:
      - ELASTICSEARCH_HOSTS=http://es:9200
    command: ["--strict.perms=false"]   # 讓掛載的檔案權限不至於擋住啟動

  cpp-app:
    build:
      context: ./app-cpp
    image: cpp-spdlog-demo:latest
    container_name: cpp-app
    depends_on:
      - filebeat
    volumes:
      - /c/Users/Asus/steven/elk-lab/logs:/logs
    environment: []
    restart: unless-stopped

volumes:
  esdata:

filebeat.yml 這邊會透過 filebeat 這個套件將日誌拉給 logstash 使用

filebeat.inputs:
  - type: filestream
    id: cpp_app_logs
    enabled: true
    paths:
      - /logs/*.log
    # 每行都是 JSON，就用 ndjson 解析；target 設空字串代表展平成根層欄位
    parsers:
      - ndjson:
          target: ""
          overwrite_keys: true
    processors:
      # 你的 JSON 若有毫秒時間戳欄位（例如 ts），可自動轉成 @timestamp
      - timestamp:
          field: ts
          layouts:
            - UNIX_MS
          ignore_missing: true
      # 打一些固定標籤方便在 Kibana 篩選
      - add_fields:
          target: labels
          fields:
            app: cpp-spdlog
            env: dev

# 出口（用 compose 的環境變數；沒設就走 elasticsearch:9200）
output.elasticsearch:
  hosts: ["${ELASTICSEARCH_HOSTS:elasticsearch:9200}"]

#（可選）讓 setup 流程能連到 Kibana
setup.kibana:
  host: "kibana:5601"

# 記錄器
logging.to_files: false
logging.level: info

app-cpp/Dockerfile

FROM ubuntu:22.04

# 確保 non-interactive 模式，避免安裝過程卡住
ENV DEBIAN_FRONTEND=noninteractive

# 更新並安裝必要工具
RUN apt-get update && \
    apt-get install -y --no-install-recommends \
        build-essential \
        git \
        clang-format \
        cppcheck \
        aspell \
        colordiff \
        zip unzip \
        tar \
        pkg-config \
        vim \
        curl \
        gdb \
        htop \
        iproute2 \
        iputils-ping \
        valgrind && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*

RUN apt-get update && \
    apt-get install -y ca-certificates && \
    cd /usr/local/src && \
    curl -LO https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.tar.gz && \
    tar -xvf cmake-3.22.2-linux-x86_64.tar.gz && \
    mv cmake-3.22.2-linux-x86_64 /usr/local/cmake

# 設定 CMake 到 PATH（對所有 shell 生效）
ENV PATH="/usr/local/cmake/bin:$PATH"

# 安裝 vcpkg
WORKDIR /opt
RUN git clone https://github.com/microsoft/vcpkg.git && \
    cd vcpkg && \
    ./bootstrap-vcpkg.sh -disableMetrics

# 設定環境變數（方便後續使用）
ENV VCPKG_ROOT=/opt/vcpkg
ENV PATH="${VCPKG_ROOT}:${PATH}"

RUN /opt/vcpkg/vcpkg install spdlog

WORKDIR /root/cpp-spdlog
COPY ./cpp-spdlog /root/cpp-spdlog
RUN mkdir build && cd build && \ 
    cmake .. -DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake && \
    make

CMD ["/root/cpp-spdlog/build/app"]

app-cpp/cpp-spdlog/main.cpp

#include <spdlog/spdlog.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#include <chrono>
#include <thread>
#include <string>

int main() {
    auto logger = spdlog::stdout_color_mt("console");
    // 時間用 epoch 毫秒，字段有 level/name/msg
    spdlog::set_pattern(R"({"ts":%Ems,"level":"%l","logger":"%n","msg":"%v"})");

    int i = 0;
    while (true) {
        logger->info("hello from spdlog #{}, user=alice, action=login", i++);
        logger->warn("warn event #{}, module=auth, code=1001", i);
        logger->error("error event #{}, module=payment, code=42", i);
        std::this_thread::sleep_for(std::chrono::milliseconds(250));
    }
    return 0;
}

app-cpp/cpp-spdlog/CMakeLists.txt

cmake_minimum_required(VERSION 3.20)
project(grpc_cpp_demo LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

find_package(spdlog CONFIG REQUIRED)
add_executable(app main.cpp)
target_link_libraries(app PRIVATE spdlog::spdlog)